package sports

import (
	"crypto/tls"
	"github.com/gocolly/colly"
	"longmen/server/config/global"
	"net"
	"net/http"
	"time"
)

var SpiderService = &spiderService{}

type spiderService struct {
	//boniuChapterListCollector *colly.Collector
}

/**
生成一个collector对象
*/
func (this *spiderService) NewCollector(domain, fullUrl string) *colly.Collector {
	collector := colly.NewCollector(
		colly.AllowedDomains(domain, fullUrl),
		colly.UserAgent("Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36"),
	)

	collector.WithTransport(&http.Transport{
		Proxy: http.ProxyFromEnvironment,
		DialContext: (&net.Dialer{
			Timeout:   90 * time.Second,
			KeepAlive: 90 * time.Second,
			DualStack: true,
		}).DialContext,
		MaxIdleConns:          100,
		IdleConnTimeout:       90 * time.Second,
		TLSHandshakeTimeout:   90 * time.Second,
		ExpectContinueTimeout: 90 * time.Second,
		TLSClientConfig:       &tls.Config{InsecureSkipVerify: true},
	})
	//
	////是否允许相同url重复请求
	collector.AllowURLRevisit = true
	//
	////默认是同步,配置为异步,这样会提高抓取效率
	//collector.Async = true
	//
	collector.DetectCharset = true
	//
	//// 对于匹配的域名(当前配置为任何域名),将请求并发数配置为2
	////通过测试发现,RandomDelay参数对于同步模式也生效
	if err := collector.Limit(&colly.LimitRule{
		// glob模式匹配域名
		DomainGlob: "*httpbin.*",
		// 匹配到的域名的并发请求数
		Parallelism: 1,
		// 在发起一个新请求时的随机等待时间
		RandomDelay: time.Duration(10) * time.Second,
	}); err != nil {
		global.Log.Error("生成一个collector对象, 限速配置失败", err)
	}

	//配置反爬策略(设置ua和refer扩展)
	//collector.RandomUserAgent(collector)
	//extensions.Referer(collector)

	return collector
}
